# encoding=utf8
import json
import math
import codecs
from pymongo import MongoClient
from bson import json_util
from urllib.parse import quote_plus
from tqdm import tqdm


user = 'mongouser'
pwd = 'tuanjie123'
host = '118.25.106.14'
port = 27017
url = 'mongodb://%s:%s@%s' % (quote_plus(user), quote_plus(pwd), host)

client = MongoClient(url, port=port)
collection = client['tagging_test']['task_test']

# 文本匹配
lines = collection.find({'taskKey': 'textMatch', 'canOutput': 1, 'checkDate': {'$gte': 1628265600000, '$lt': 1636686000000}},
                        {'checkData': 1, 'reCheckData': 1, 'finalData': 1, 'checkDate': 1, '_id': 0})


# raise Exception('ting')
# total = collection.count_documents({'taskKey': 'textMatch', 'canOutput': 1, 'checkDate': {'$gte': 1636300800000}})  # 数据总量
total = lines.count()
# print(total)
# raise Exception("ting")
# print(linez.count())
cut_num = 10000  # 数据量的10%村一个文件
file_index = 1  # 文件标志

save_list = []
success_count = 0

for idx, line in tqdm(enumerate(lines), total=total):  # lines是动态读取mongo中的数据
    if (idx + 1) % cut_num == 0:  # 达到存储要求
        # 将数据写入文件
        with codecs.open('./fl_json_file/json_' + str(file_index) + '.json', 'w', encoding='utf-8') as f:
            json.dump(save_list, f, ensure_ascii=False, indent=4)
        # 清空列表
        save_list = []
        # 文件标志加一
        file_index += 1
        last_save = idx

    # print(line)
    # raise Exception('ting')
    # js = json.util.dumps(line)  # 处理_id字段
    # print(line)
    # raise Exception('ting')
    # print(type(line))
    save_list.append(line)
    success_count += 1

# 判断是否有多余数据
if save_list:
    with codecs.open('./htpp_json_file/json_' + str(file_index) + '.json', 'w', encoding='utf-8') as f:
        json.dump(save_list, f, ensure_ascii=False, indent=4)

print(success_count)

